require 'yaml'  # ruby stdlib
require 'zlib'  # ruby stdlib

require 'sitemap_spider/crawl'
require 'sitemap_spider/options'
require 'sitemap_spider/sitemap_record'
require 'sitemap_spider/sitemap_image_record'
require 'sitemap_spider/url'
require 'sitemap_spider/xml'

# Add +referenced_from+ and +orig_href_string+ instance attributes to URI::Generic
class URI::Generic
  attr_accessor :referenced_from, :orig_href_string
end

module SitemapSpider

  class Sitemap
    def initialize(opts = nil)
      @opts = (opts ? opts : self.class.default_opts)

      # holds pages to be crawled, pages crawled (these go to the map), pages which failed to be downloaded
      @pages_to_crawl = []
      @pages_crawled = []
      @pages_failed_download = []
    end

    def inspect_state(only_size = false)
      $stdout.print("INFO: Pages state\n")
      $stdout.printf("to_crawl: %s\n", @pages_to_crawl.size)
      @pages_to_crawl.each {|smrec| $stdout.printf("%s\n", smrec.loc)} unless only_size
      $stdout.printf("crawled: %s\n", @pages_crawled.size)
      @pages_crawled.each {|smrec| $stdout.printf("%s\n", smrec.loc)} unless only_size
      $stdout.printf("failed download: %s\n", @pages_failed_download.size)
      @pages_failed_download.each {|smrec| $stdout.printf("%s\n", smrec.loc)} unless only_size
      $stdout.flush
    end

    def remove_sitemap
      exclusive_lock
      File.delete(@opts[:output_file])
      ensure
        unlock
    end

    def compress_sitemap
      exclusive_lock
      Zlib::GzipWriter.open(@opts[:output_file] + '.gz') do |gz|
        gz.write(File.read(@opts[:output_file]))
      end
      ensure
        unlock
    end

    def sitemap_from_links
      @pages_to_crawl = File.readlines(@opts[:input_file]).collect {|line| SitemapRecord.new(line.chomp)}
      sitemap_from_whatever
    end

    def sitemap_from_site
      if @opts[:produce_sitemap]
        exclusive_lock
        get_sitemap_document
      end
      popshift = (@opts[:process_order_is_fifo] ? :shift : :pop)
      crawl_for_links(SitemapRecord.new(normalize_port!(URI::Generic.build(:scheme => @opts[:protocol],
        :host => @opts[:host], :port => @opts[:port], :path => @opts[:start_path]))))

      # crawl each page in pages array unless it's already been crawled
      while @pages_to_crawl.size > 0 do
        crawl_for_links(@pages_to_crawl.send(popshift))
        sleep @opts[:sleep_between_requests]
      end
      if @opts[:produce_sitemap]
        update_sitemap_xml(@pages_crawled)
        write_out_xml
      end
      inspect_state(true) if @opts[:print_totals]
      ensure
        unlock
    end

    def sitemap_from_yaml
      @pages_to_crawl = YAML.load(File.open(@opts[:input_file])).collect {|hsh| SitemapRecord.new(hsh)}
      sitemap_from_whatever
    end

    # add records, don't delete duplicates (cheap)
    def add_records(a_rec)
      if @opts[:produce_sitemap]
        exclusive_lock
        get_sitemap_document
        a_smrecs = (a_rec ? a_rec.collect {|r| SitemapRecord.new(r)} : [])
        update_sitemap_xml(a_smrecs)
        write_out_xml
      end
      ensure
        unlock
    end

    # first delete all d_records, then add a_records
    # XXX adding/updating records is very expensive, because we try to find and delete duplicates
    def update_records(a_rec, d_rec)
      if @opts[:produce_sitemap]
        exclusive_lock
        get_sitemap_document
        a_smrecs = (a_rec ? a_rec.collect {|r| SitemapRecord.new(r)} : [])
        d_smrecs = (d_rec ? d_rec.collect {|r| SitemapRecord.new(r)} : [])
        update_sitemap_xml(d_smrecs, :delete => true)
        update_sitemap_xml(a_smrecs, :delete_duplicates => true)
        write_out_xml
      end
      ensure
        unlock
    end

    private

    def sitemap_from_whatever
      if @opts[:produce_sitemap]
        exclusive_lock
        get_sitemap_document
      end
      if @opts[:add_only_valid]
        go_through_links
        for_xml_update = @pages_crawled
      else
        for_xml_update = @pages_to_crawl
      end
      if @opts[:produce_sitemap]
        update_sitemap_xml(for_xml_update)
        write_out_xml
      end
      inspect_state(true) if @opts[:print_totals]
      ensure
        unlock
    end

    def exclusive_lock
      @lock = File.open(@opts[:lock_file], File::CREAT|File::WRONLY)
      @lock.flock(File::LOCK_EX)
    end

    def unlock
      @lock.flock(File::LOCK_UN) rescue nil
      @lock.close rescue nil
    end
  end
end
